# -*- coding: utf-8 -*-
import scrapy
from scrapy.spiders import Rule
from scrapy.linkextractors import LinkExtractor
import re
from scrapyTest.items import PageItem


class FlaskSpider(scrapy.spiders.CrawlSpider):
  name = 'flask'
  allowed_domains = ['flask.pocoo.org']
  start_urls = ["http://flask.pocoo.org/docs/0.12/"]
  le = LinkExtractor(allow="flask.pocoo.org/d+",restrict_css='div#user-s-guide a.reference')
  rules = (
      Rule(link_extractor=le,callback='parse_page',follow=True),
  )

  def parse_page(self, response):
      item = PageItem()
      """
      TODO:补充 url 和 text 的解析规则
      """
      item['url'] = response.url
      html = response.xpath('//body').extract_first()
      html = re.sub(r'<[^>]+>',"",html)
      item['text'] = re.sub(r'\s+',' ', html)
      yield item
